package com.SparkCore.RDD.persist

import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

/**
 * 持久化
 * 实现wordCount
 */
object Spark02_RDD_Persist {
  def main(args: Array[String]): Unit = {
    //建立和Spark框架的链接
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
    val sc: SparkContext = new SparkContext(sparkConf)

    //创建RDD算子
    val path = "datas/1.txt"
    val txt: RDD[String] = sc.textFile(path)

    val flatRDD: RDD[String] = txt.flatMap(_.split(" "))

    val mapRDD: RDD[(String, Int)] = flatRDD.map((_, 1))

    //缓存
    mapRDD.cache()
    mapRDD.persist(StorageLevel.DISK_ONLY)
    /**
     * cache默认持久化的操作，只能将数据保存到内存中，如果想要保存到磁盘文件 就用persist来指定位置
     */

    val reduceRDD: RDD[(String, Int)] = mapRDD.reduceByKey(_+_)

    reduceRDD.collect().foreach(println)

    println("====================")

    //持久化体现
    //把重复的去掉，可以重用的都使用起来
    val reduceRDD1: RDD[(String, Iterable[Int])] = reduceRDD.groupByKey()

    reduceRDD1.collect().foreach(println)
    //关闭链接
    sc.stop()
  }
}
